package com.coderman;

import com.alibaba.fastjson.JSON;
import com.coderman.model.AreaConfig;
import com.coderman.model.AreaConfigModel;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicReference;

/**
 * description: Start <br>
 * date: 2020/9/12 15:36 <br>
 * author: coderman <br>
 * version: 1.0 <br>
 */
public class CityAreaSpider implements PageProcessor {

    private Object object = new Object();
    private Site site = Site.me()
            .setRetryTimes(4)
            .setSleepTime(2000)
            .setCharset("UTF-8");
    static ConcurrentHashMap<String, CopyOnWriteArrayList<AreaConfig>> allURlMap = new ConcurrentHashMap<>();
    static ConcurrentHashMap<String, String> allMetaMap = new ConcurrentHashMap<>();

    BlockingQueue<String> areaDataList = new ArrayBlockingQueue<String>(10000000);

    public static void main(String[] args) {

        long startTime = System.currentTimeMillis();

        AreaSpider areaSpider = new AreaSpider();
        allURlMap = areaSpider.getAllURLMap();

        Set<String> set = new HashSet<>();
        allURlMap.forEach((k, v) -> {
            if (v != null) {
                v.forEach(areaConfig -> {
                    if (areaConfig.getUrlMap() != null) {
                        areaConfig.getUrlMap().forEach((m, l) -> {
                            for (int i = 1; i <= 100; i++) {
                                String x = l + "p" + i + "/";
                                set.add(x);
                            }

                            allMetaMap.put(l, k + "," + areaConfig.getCityName() + "," + m);
                        });
                    }

                });
            }
        });


        String[] arr = new String[set.size()];
        set.toArray(arr);

        System.out.println(JSON.toJSONString(allURlMap));
        System.out.println(JSON.toJSONString(allMetaMap));

        Spider.create(new CityAreaSpider()).thread(30).addUrl(arr)
                .run();
        long endTime = System.currentTimeMillis();

        System.out.println("======useTime = "+(endTime -startTime)/1000+"s.");

       /* allURlMap.forEach((k, v) -> {
            v.forEach(areaConfig -> {
                if (areaConfig.getAreaListMap() != null) {
                    areaConfig.getAreaListMap().forEach((s, areaConfigModels) -> {
                        areaConfigModels.forEach(areaConfigModel -> {
                            //注意线程安全问题
                            StringBuffer builder = new StringBuffer();
                            builder.append(k).append(",")
                                    .append(areaConfig.getCityName()).append(",")
                                    .append(s).append(",")
                                    .append(areaConfigModel.getAreaName()).append(",")
                                    .append(areaConfigModel.getAreaAddress());
                            System.out.println(builder.toString());
                           // areaDataList.add(builder.toString());
                        });
                    });
                }
            });
        });*/


      /*  try {

            int size = areaDataList.size();
            int page = size / 1000000;

            for (int i = 0;i < page;i++){
                File file = new File("E:\\轻蜗牛直租平台项目\\中国所有小区\\allarea_"+System.currentTimeMillis()+"_"+i+".txt");
                FileUtils.writeLines(file, areaDataList.subList(i*1000000,(i+1)*1000000));
            }

            if(size % 1000000 != 0 ){
                File file = new File("E:\\轻蜗牛直租平台项目\\中国所有小区\\allarea_"+System.currentTimeMillis()+"_"+(page+1)+".txt");

                FileUtils.writeLines(file, areaDataList.subList((page)*1000000,size));

            }
            long endTime = System.currentTimeMillis();
            System.out.println("useTime = "+(endTime - startTime)/1000+"s");


        } catch (IOException e) {
            e.printStackTrace();
        }*/
    }

    @Override
    public void process(Page page) {
        String url = page.getUrl().get();
        if (areaDataList.size() > 10000) {
            CopyOnWriteArrayList<String> copyOnWriteArrayList = new CopyOnWriteArrayList<>();
            try {
                for (int i = 0; i < 10000; i++) {
                    copyOnWriteArrayList.add(areaDataList.take());
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            File file = new File("E:\\轻蜗牛直租平台项目\\中国所有小区\\allarea_" + System.currentTimeMillis() + ".txt");
            try {
                FileUtils.writeLines(file, copyOnWriteArrayList);
                System.out.println("write data success ============ " + file.getName());
            } catch (IOException e) {
                e.printStackTrace();
            }
            copyOnWriteArrayList.clear();
        }

        try {
            for (int i = 1; i <= 25; i++) {
                String x = page.getHtml().xpath("/html/body/div[3]/div/div[2]/div[1]/ul/li[" + i + "]/div/h2/a/text()").all().get(0);
                String d = page.getHtml().xpath("/html/body/div[3]/div/div[2]/div[1]/ul/li[" + i + "]/div/div[1]/div[2]/span/text()").all().get(0);
                String t = page.getHtml().xpath("/html/body/div[3]/div/div[2]/div[1]/ul/li[" + i + "]/div/div[1]/div[3]/span/text()").all().get(0);
                String key = url.substring(0, url.length() - 1);
                key = key.substring(0, key.lastIndexOf("/")) + "/";
                StringBuffer buffer = new StringBuffer(allMetaMap.get(key));
                buffer.append(",").append(x).append(",").append(d).append(",").append(t);
                areaDataList.put(buffer.toString());
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        ///AtomicReference<String> areaName = new AtomicReference<>("");

/*
        allURlMap.forEach((k, v) -> {
            v.forEach(areaConfig -> {
                areaConfig.getUrlMap().forEach((x, y) -> {
                    if (url.contains(y)) {
                        areaName.set(x);
                    }
                });
            });

            if (areaName.get().equals("") || areaName.get() == null) {
                v.forEach(areaConfigx -> {
                    areaConfigx.getUrlMap().forEach((x, y) -> {
                        if (url.contains(y)) {
                            areaName.set(x);
                        }
                    });
                });
            }
        });
*/



/*
        allURlMap.forEach((k, v) -> {
            v.forEach(areaConfig -> {
                ConcurrentHashMap<String, CopyOnWriteArrayList<AreaConfigModel>> areaConfigModelMap = new ConcurrentHashMap<>();

                if (areaConfig.getAreaListMap() == null) {
                    areaConfig.setAreaListMap(areaConfigModelMap);
                    CopyOnWriteArrayList<AreaConfigModel> oldList = areaConfig.getAreaListMap().get(areaName.get());
                    if (oldList == null) {
                        areaConfigModelMap.put(areaName.get(), areaConfigModelList);
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    } else {
                        oldList.addAll(areaConfigModelList);
                        areaConfigModelMap.put(areaName.get(), oldList);
                        areaConfigModelMap.putAll(areaConfig.getAreaListMap());
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    }
                } else {
                    CopyOnWriteArrayList<AreaConfigModel> oldList = areaConfig.getAreaListMap().get(areaName.get());
                    if (oldList == null) {
                        areaConfigModelMap.put(areaName.get(), areaConfigModelList);
                        areaConfigModelMap.putAll(areaConfig.getAreaListMap());
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    } else {
                        oldList.addAll(areaConfigModelList);
                        areaConfigModelMap.put(areaName.get(), oldList);
                        areaConfigModelMap.putAll(areaConfig.getAreaListMap());
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    }
                }

            });
        });
*/


/*
        allURlMap.forEach((k, v) -> {
            v.forEach(areaConfig -> {
                AtomicReference<String> areaName = new AtomicReference<>("");
                areaConfig.getUrlMap().forEach((x, y) -> {
                    if (url.contains(y)) {
                        areaName.set(x);
                    }
                });
                if (areaName.get().equals("") || areaName.get() == null) {
                    v.forEach(areaConfigx -> {
                        areaConfigx.getUrlMap().forEach((x, y) -> {
                            if (url.contains(y)) {
                                areaName.set(x);
                            }
                        });
                    });
                }
                ConcurrentHashMap<String, List<AreaConfigModel>> areaConfigModelMap = new ConcurrentHashMap<>();

                if (StringUtils.isEmpty(areaName.get())) {
                    System.out.println(JSON.toJSONString(areaConfig.getUrlMap()) + "," + url + "," + areaConfig.getCityUrl() + "," + areaConfig.getCityName());
                }
                if (areaConfig.getAreaListMap() == null) {
                    areaConfig.setAreaListMap(areaConfigModelMap);
                    List<AreaConfigModel> oldList = areaConfig.getAreaListMap().get(areaName.get());
                    if (oldList == null) {
                        areaConfigModelMap.put(areaName.get(), areaConfigModelList);
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    } else {
                        oldList.addAll(areaConfigModelList);
                        areaConfigModelMap.put(areaName.get(), oldList);
                        areaConfigModelMap.putAll(areaConfig.getAreaListMap());
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    }
                } else {
                    List<AreaConfigModel> oldList = areaConfig.getAreaListMap().get(areaName.get());
                    if (oldList == null) {
                        areaConfigModelMap.put(areaName.get(), areaConfigModelList);
                        areaConfigModelMap.putAll(areaConfig.getAreaListMap());
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    } else {
                        oldList.addAll(areaConfigModelList);
                        areaConfigModelMap.put(areaName.get(), oldList);
                        areaConfigModelMap.putAll(areaConfig.getAreaListMap());
                        areaConfig.setAreaListMap(areaConfigModelMap);
                    }
                }

            });
        });
*/
    }

    @Override
    public Site getSite() {
        return site;
    }

}
